<!DOCTYPE html>












  


<html class="theme-next pisces use-motion" lang="zh-CN">
<head>
  <meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2"/>
<meta name="theme-color" content="#222">












<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />



















  
  
  
  

  
    
    
  

  

  

  

  
    
      
    

    
  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic|consolas:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=6.3.0" rel="stylesheet" type="text/css" />


  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png?v=6.3.0">


  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png?v=6.3.0">


  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png?v=6.3.0">


  <link rel="mask-icon" href="/images/logo.svg?v=6.3.0" color="#222">









<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Pisces',
    version: '6.3.0',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":false,"scrollpercent":false,"onmobile":false},
    fancybox: false,
    fastclick: false,
    lazyload: false,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>


  




  <meta name="description" content="安装ubuntu在windows上使用ultraISO软件来刻录Ubuntu系统安装盘，设置U盘启动，安装系统参考链接：Ubuntu 14.04 64位系统安装cuda8.0+cudnn7.5+opencv+caffe 血泪教程 安装NVIDIA驱动http://www.nvidia.cn/Download/index.aspx?lang=cn选择合适型号，下载驱动包  首先按Ctrl+Alt+F">
<meta name="keywords" content="caffe,深度学习,大数据,cuda,opencv,分布式">
<meta property="og:type" content="article">
<meta property="og:title" content="分布式大数据分析平台环境搭建">
<meta property="og:url" content="https://xiongraorao.github.io/2017/03/07/大数据分析平台环境搭建/index.html">
<meta property="og:site_name" content="在路上">
<meta property="og:description" content="安装ubuntu在windows上使用ultraISO软件来刻录Ubuntu系统安装盘，设置U盘启动，安装系统参考链接：Ubuntu 14.04 64位系统安装cuda8.0+cudnn7.5+opencv+caffe 血泪教程 安装NVIDIA驱动http://www.nvidia.cn/Download/index.aspx?lang=cn选择合适型号，下载驱动包  首先按Ctrl+Alt+F">
<meta property="og:locale" content="zh-CN">
<meta property="og:updated_time" content="2018-06-08T09:10:07.352Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="分布式大数据分析平台环境搭建">
<meta name="twitter:description" content="安装ubuntu在windows上使用ultraISO软件来刻录Ubuntu系统安装盘，设置U盘启动，安装系统参考链接：Ubuntu 14.04 64位系统安装cuda8.0+cudnn7.5+opencv+caffe 血泪教程 安装NVIDIA驱动http://www.nvidia.cn/Download/index.aspx?lang=cn选择合适型号，下载驱动包  首先按Ctrl+Alt+F">



  <link rel="alternate" href="/atom.xml" title="在路上" type="application/atom+xml" />




  <link rel="canonical" href="https://xiongraorao.github.io/2017/03/07/大数据分析平台环境搭建/"/>



<script type="text/javascript" id="page.configurations">
  CONFIG.page = {
    sidebar: "",
  };
</script>

  <title>分布式大数据分析平台环境搭建 | 在路上</title>
  




<script async src="https://www.googletagmanager.com/gtag/js?id=UA-120522271-1"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());

  gtag('config', 'UA-120522271-1');
</script>






  <noscript>
  <style type="text/css">
    .use-motion .motion-element,
    .use-motion .brand,
    .use-motion .menu-item,
    .sidebar-inner,
    .use-motion .post-block,
    .use-motion .pagination,
    .use-motion .comments,
    .use-motion .post-header,
    .use-motion .post-body,
    .use-motion .collection-title { opacity: initial; }

    .use-motion .logo,
    .use-motion .site-title,
    .use-motion .site-subtitle {
      opacity: initial;
      top: initial;
    }

    .use-motion {
      .logo-line-before i { left: initial; }
      .logo-line-after i { right: initial; }
    }
  </style>
</noscript>

  
    <script type="text/javascript" src="//cdn.bootcss.com/canvas-nest.js/1.0.0/canvas-nest.min.js"></script>
  
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-CN">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta custom-logo">
    

    <div class="custom-logo-site-title">
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">在路上</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
    
      
        <h1 class="site-subtitle" itemprop="description">stay hungry, stay foolish!</h1>
      
    
  </div>

  <div class="site-nav-toggle">
    <button aria-label="切换导航栏">
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>



<nav class="site-nav">
  
    <ul id="menu" class="menu">
      
        
        
        
          
          <li class="menu-item menu-item-home">
    <a href="/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-home"></i> <br />首页</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-about">
    <a href="/about/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-user"></i> <br />关于</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-tags">
    <a href="/tags/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />标签</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-categories">
    <a href="/categories/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-th"></i> <br />分类</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-archives">
    <a href="/archives/" rel="section">
      <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />归档</a>
  </li>
        
        
        
          
          <li class="menu-item menu-item-sitemap">
    <a href="/sitemap.xml" rel="section">
      <i class="menu-item-icon fa fa-fw fa-sitemap"></i> <br />站点地图</a>
  </li>

      
      
        <li class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />搜索</a>
        </li>
      
    </ul>
  

  

  
    <div class="site-search">
      
  <div class="popup search-popup local-search-popup">
  <div class="local-search-header clearfix">
    <span class="search-icon">
      <i class="fa fa-search"></i>
    </span>
    <span class="popup-btn-close">
      <i class="fa fa-times-circle"></i>
    </span>
    <div class="local-search-input-wrapper">
      <input autocomplete="off"
             placeholder="搜索..." spellcheck="false"
             type="text" id="local-search-input">
    </div>
  </div>
  <div id="local-search-result"></div>
</div>



    </div>
  
</nav>



  



</div>
    </header>

    
  
  
  
    
      
    
    <a href="https://github.com/xiongraorao" class="github-corner" target="_blank" title="Follow me on GitHub" aria-label="Follow me on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#222; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg>
    
      </a>
    



    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          
            

          
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  
    <div class="reading-progress-bar"></div>
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="https://xiongraorao.github.io/2017/03/07/大数据分析平台环境搭建/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content="Xiong Raorao">
      <meta itemprop="description" content="用我的知识改变世界">
      <meta itemprop="image" content="/uploads/avatar.jpg">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="在路上">
    </span>

    
      <header class="post-header">

        
        
          <h2 class="post-title" itemprop="name headline">分布式大数据分析平台环境搭建
              
            
          </h2>
        

        <div class="post-meta">
          <span class="post-time">

            
            
            

            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              

              
                
              

              <time title="创建时间：2017-03-07 10:29:58" itemprop="dateCreated datePublished" datetime="2017-03-07T10:29:58+08:00">2017-03-07</time>
            

            
              

              
                
                <span class="post-meta-divider">|</span>
                

                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                
                  <span class="post-meta-item-text">更新于</span>
                
                <time title="修改时间：2018-06-08 17:10:07" itemprop="dateModified" datetime="2018-06-08T17:10:07+08:00">2018-06-08</time>
              
            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/解决方案/" itemprop="url" rel="index"><span itemprop="name">解决方案</span></a></span>

                
                
                  ，
                
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing"><a href="/categories/解决方案/hadoop/" itemprop="url" rel="index"><span itemprop="name">hadoop</span></a></span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/2017/03/07/大数据分析平台环境搭建/#comments" itemprop="discussionUrl">
                  <span class="post-meta-item-text">评论数：</span> <span class="post-comments-count gitment-comments-count" data-xid="/2017/03/07/大数据分析平台环境搭建/" itemprop="commentsCount"></span>
                </a>
              </span>
            
          

          
          
             <span id="/2017/03/07/大数据分析平台环境搭建/" class="leancloud_visitors" data-flag-title="分布式大数据分析平台环境搭建">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数：</span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

          

          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <h2 id="安装ubuntu"><a href="#安装ubuntu" class="headerlink" title="安装ubuntu"></a>安装ubuntu</h2><p>在windows上使用ultraISO软件来刻录Ubuntu系统安装盘，设置U盘启动，安装系统<br>参考链接：<a href="https://xiongraorao.github.io/2017/01/14/%E5%8E%9F-Ubuntu-14-04-64%E4%BD%8D%E7%B3%BB%E7%BB%9F%E5%AE%89%E8%A3%85cuda8-0-cudnn7-5-opencv-caffe-%E8%A1%80%E6%B3%AA%E6%95%99%E7%A8%8B/">Ubuntu 14.04 64位系统安装cuda8.0+cudnn7.5+opencv+caffe 血泪教程</a></p>
<h2 id="安装NVIDIA驱动"><a href="#安装NVIDIA驱动" class="headerlink" title="安装NVIDIA驱动"></a>安装NVIDIA驱动</h2><p><a href="http://www.nvidia.cn/Download/index.aspx?lang=cn" target="_blank" rel="noopener">http://www.nvidia.cn/Download/index.aspx?lang=cn</a><br>选择合适型号，下载驱动包</p>
<ul>
<li>首先按Ctrl+Alt+F1，进入text mode，关闭显示驱动<br> sudo service lightdm stop</li>
<li><p>增加执行权限</p>
  <figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">sudo chmod +x NVIDIA-Linux-x86_64-375.39.run</span><br><span class="line">sudo ./NVIDIA-Linux-x86_64-375.39.run</span><br></pre></td></tr></table></figure>
</li>
<li><p>安装成功后，重启图像界面服务<br> sudo service lightdm start</p>
</li>
</ul>
<h2 id="安装cuda-cudnn-opencv-caffe"><a href="#安装cuda-cudnn-opencv-caffe" class="headerlink" title="安装cuda+cudnn+opencv+caffe"></a>安装cuda+cudnn+opencv+caffe</h2><p>参考这个链接：<a href="https://xiongraorao.github.io/2017/01/14/%E5%8E%9F-Ubuntu-14-04-64%E4%BD%8D%E7%B3%BB%E7%BB%9F%E5%AE%89%E8%A3%85cuda8-0-cudnn7-5-opencv-caffe-%E8%A1%80%E6%B3%AA%E6%95%99%E7%A8%8B/">原-Ubuntu-14-04-64位系统安装cuda8-0-cudnn7-5-opencv-caffe-血泪教程</a></p>
<p>caffe_github_link:<a href="https://github.com/BVLC/caffe.git" target="_blank" rel="noopener">https://github.com/BVLC/caffe.git</a><br>opencv_github_link:<a href="https://github.com/opencv/opencv" target="_blank" rel="noopener">https://github.com/opencv/opencv</a></p>
<h2 id="配置静态ip-安装ssh-server"><a href="#配置静态ip-安装ssh-server" class="headerlink" title="配置静态ip + 安装ssh-server"></a>配置静态ip + 安装ssh-server</h2><h3 id="静态ip"><a href="#静态ip" class="headerlink" title="静态ip"></a>静态ip</h3><p>ubuntu14.04设置静态ip</p>
<ol>
<li>找到文件并作如下修改：</li>
</ol>
<p>sudo vim /etc/network/interfaces</p>
<p>修改如下部分：</p>
<p>auto eth0<br>iface eth0 inet static<br>address 192.168.1.4<br>gateway 192.168.1.1 #这个地址你要确认下 网关是不是这个地址<br>netmask 255.255.255.0</p>
<h1 id="network-192-168-0-0"><a href="#network-192-168-0-0" class="headerlink" title="network 192.168.0.0"></a>network 192.168.0.0</h1><h1 id="broadcast-192-168-0-255"><a href="#broadcast-192-168-0-255" class="headerlink" title="broadcast 192.168.0.255"></a>broadcast 192.168.0.255</h1><ol>
<li>修改dns解析</li>
</ol>
<p>因为以前是dhcp解析，所以会自动分配dns服务器地址</p>
<p>而一旦设置为静态ip后就没有自动获取到的dns服务器了</p>
<p>要自己设置一个</p>
<p>sudo vim /etc/resolv.conf</p>
<p>写上一个公网的DNS</p>
<p>nameserver 202.114.0.131(公网dns服务器)<br>nameserver 202.114.0.242（公网dns服务器）<br>nameserver 192.168.1.1（局域网内可以访问外网的机器，非网关服务器放在前面）</p>
<p>sudo vim /etc/resolvconf/resolv.d/base</p>
<p>nameserver 202.114.0.131(公网dns服务器)<br>nameserver 202.114.0.242（公网dns服务器）<br>nameserver 192.168.1.1（局域网内可以访问外网的机器，非网关服务器放在前面）</p>
<ol>
<li>重启网卡：</li>
</ol>
<p>sudo /etc/init.d/network restart</p>
<h3 id="安装ssh-server"><a href="#安装ssh-server" class="headerlink" title="安装ssh-server"></a>安装ssh-server</h3><ul>
<li>判断是否安装ssh服务<br>Ubuntu系统默认安装ssh-client，如果想远程登录主机，就需要安装ssh-server，如下命令：  <figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ps -e|grep sshps -e|grep ssh</span><br></pre></td></tr></table></figure>
</li>
</ul>
<p>ssh-agent表示ssh-client启动，sshd表示ssh-server启动了</p>
<p>如果缺少sshd，说明ssh服务没有启动或者没有安装。<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">sudo apt-get install openssh-client <span class="comment">#安装ssh-client命令</span></span><br><span class="line">sudo apt-get install openssh-server <span class="comment">#安装ssh-server命令</span></span><br><span class="line">sudo /etc/init.d/ssh start <span class="comment">#启动服务</span></span><br><span class="line">ps -e|grep sshps -e|grep ssh <span class="comment">#查看是否正确启动。</span></span><br></pre></td></tr></table></figure></p>
<ul>
<li><p>配置ssh服务端口<br>默认端口是 22<br>sudo gedit /etc/ssh/sshd_config<br>sudo /etc/init.d/ssh restart #重启ssh服务生效</p>
</li>
<li><p>设置开机自动启动</p>
</li>
</ul>
<h2 id="安装storm-kafka-zookeeper-hdfs-hbase-jdk"><a href="#安装storm-kafka-zookeeper-hdfs-hbase-jdk" class="headerlink" title="安装storm+kafka+zookeeper+hdfs+hbase+jdk"></a>安装storm+kafka+zookeeper+hdfs+hbase+jdk</h2><h3 id="前期准备工作"><a href="#前期准备工作" class="headerlink" title="前期准备工作"></a>前期准备工作</h3><ol>
<li>首先下载好这几个文件，这里用的版本分别是：</li>
</ol>
<ul>
<li>storm <strong><em>0.9.6</em></strong></li>
<li>zookeeper <strong><em>3.4.0</em></strong></li>
<li>jdk <strong><em>1.8.0.92</em></strong></li>
<li>kafka <strong><em>2.11.0.10.0.0</em></strong></li>
<li>hadoop <strong><em>2.6.4</em></strong></li>
<li>hbase <strong><em>1.2.4</em></strong></li>
</ul>
<ol>
<li>设置ip地址映射<br>sudo vi /etc/hosts</li>
</ol>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># test</span></span><br><span class="line">192.168.0.136   pc1080</span><br><span class="line">192.168.0.116   pcwrp</span><br><span class="line">192.168.0.117   pcxrr</span><br><span class="line"></span><br><span class="line">192.168.0.136   cloud01</span><br><span class="line">192.168.0.116   cloud02</span><br><span class="line">192.168.0.117   cloud03</span><br><span class="line"></span><br><span class="line">192.168.0.136   hadoop01</span><br><span class="line">192.168.0.116   hadoop02</span><br><span class="line">192.168.0.117   hadoop03</span><br><span class="line"></span><br><span class="line">192.168.0.136   zk01</span><br><span class="line">192.168.0.116   zk02</span><br><span class="line">192.168.0.117   zk03</span><br><span class="line"></span><br><span class="line">192.168.0.136   kafka01</span><br><span class="line">192.168.0.116   kafka02</span><br><span class="line">192.168.0.117   kafka03</span><br><span class="line"></span><br><span class="line">192.168.0.136   kafka01</span><br><span class="line">192.168.0.116   kafka02</span><br><span class="line">192.168.0.117   kafka03</span><br><span class="line"></span><br><span class="line">192.168.0.136   storm01</span><br><span class="line">192.168.0.116   storm02</span><br><span class="line">192.168.0.117   storm03</span><br><span class="line"></span><br><span class="line">192.168.0.136   hbase01</span><br><span class="line">192.168.0.116   hbase02</span><br><span class="line">192.168.0.117   hbase03</span><br><span class="line"></span><br><span class="line"><span class="comment"># end test</span></span><br></pre></td></tr></table></figure>
<p><span style="color:red">这里面需要注意一下，之所以需要配置多组ip映射，是因为在相应的软件中很多默认参数需要用到这些映射</span></p>
<ol>
<li><p>三台机机器实现互相ssh免密码登录，并且实现自身环回登录<br>参考链接：<a href="https://xiongraorao.github.io/2017/02/17/ssh%E5%85%8D%E5%AF%86%E7%A0%81%E7%99%BB%E5%BD%95/">ssh免密码登录</a></p>
</li>
<li><p>设置环境变量<br>vi ~/.bashrc</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># java</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/home/hadoop/cloud/jdk1.8.0_92/</span><br><span class="line"><span class="built_in">export</span> JRE_HOME=<span class="variable">$JAVA_HOME</span>/jre</span><br><span class="line"><span class="built_in">export</span> CLASSPATH=.:<span class="variable">$JAVA_HOME</span>/lib/tools.jar:<span class="variable">$JAVA_HOME</span>/lib/dt.jar:<span class="variable">$JRE_HOME</span>/lib</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$PATH</span>:<span class="variable">$JAVA_HOME</span>/bin:<span class="variable">$JRE_HOME</span>/bin</span><br><span class="line"><span class="comment"># hadoop</span></span><br><span class="line"><span class="built_in">export</span> MAVEN_HOME=/home/hadoop/cloud/apache-maven-3.3.9/</span><br><span class="line"><span class="built_in">export</span> ZOOKEEPER_HOME=/home/hadoop/cloud/zookeeper-3.4.9/</span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/home/hadoop/cloud/hadoop-2.6.4/</span><br><span class="line"><span class="built_in">export</span> HBASE_HOME=/home/hadoop/cloud/hbase-1.2.4/</span><br><span class="line"><span class="built_in">export</span> FLUME_HOME=/home/hadoop/cloud/apache-flume-1.6.0/</span><br><span class="line"><span class="built_in">export</span> KAFKA_HOME=/home/hadoop/cloud/kafka_2.11-0.10.0.0/</span><br><span class="line"><span class="built_in">export</span> STORM_HOME=/home/hadoop/cloud/apache-storm-0.9.6/</span><br><span class="line"><span class="built_in">export</span> HIVE_HOME=/home/hadoop/cloud/apache-hive-1.2.1/</span><br><span class="line"><span class="built_in">export</span> HADOOP_INSTALL=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_PREFIX=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_MAPRED_HOME=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_COMMON_HOME=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_HDFS_HOME=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> YARN_HOME=<span class="variable">$HADOOP_HOME</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_COMMON_LIB_NATIVE_DIR=<span class="variable">$HADOOP_HOME</span>/lib/native</span><br><span class="line"><span class="built_in">export</span> HADOOP_OPTS=<span class="string">"-Djava.library.path=<span class="variable">$HADOOP_PREFIX</span>/lib/native"</span></span><br><span class="line"><span class="comment">#export CHUKWA_HOME=/home/hadoop/cloud/chukwa-0.8.0/</span></span><br><span class="line"><span class="comment">#export CHUKWA_CONF_DIR=$CHUKWA_HOME/etc/chukwa</span></span><br><span class="line">PATH=<span class="variable">$PATH</span>:<span class="variable">$HOME</span>/bin</span><br><span class="line">PATH=<span class="variable">$JAVA_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$MAVEN_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$ZOOKEEPER_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$HBASE_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$FLUME_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$KAFKA_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$STORM_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$PATH</span>:<span class="variable">$HADOOP_HOME</span>/sbin:<span class="variable">$HADOOP_HOME</span>/bin</span><br><span class="line">PATH=<span class="variable">$HIVE_HOME</span>/bin/:<span class="variable">$PATH</span></span><br><span class="line">PATH=<span class="variable">$PATH</span>:<span class="variable">$CHUKWA_HOME</span>/bin:<span class="variable">$CHUKWA_HOME</span>/sbin</span><br><span class="line"><span class="built_in">export</span> PATH</span><br></pre></td></tr></table></figure>
</li>
</ol>
<h3 id="安装zookeeper"><a href="#安装zookeeper" class="headerlink" title="安装zookeeper"></a>安装zookeeper</h3><ul>
<li>下载安装包，解压到 /home/hadoop/cloud 目录下</li>
<li>添加环境变量</li>
<li><strong>配置/etc/hosts 文件，做好ip地址映射</strong></li>
<li>cat zoo_sample.cfg &gt;&gt; zoo.cfg</li>
<li><p>编辑 zoo.cfg文件</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">tickTime=2000</span><br><span class="line">initLimit=10</span><br><span class="line">syncLimit=5</span><br><span class="line"><span class="comment"># zookeeper数据保存路径</span></span><br><span class="line">dataDir=/home/hadoop/cloud/zookeeper-3.4.9/data/</span><br><span class="line">clientPort=2181</span><br><span class="line">maxClientCnxns=150</span><br><span class="line">maxSessionTimeout=100000</span><br><span class="line">autopurge.snapRetainCount=6</span><br><span class="line">autopurge.purgeInterval=48</span><br><span class="line">server.1=zk01:2888:3888</span><br><span class="line">server.2=zk02:2888:3888</span><br><span class="line">server.3=zk03:2888:3888</span><br></pre></td></tr></table></figure>
</li>
<li><p>在节点配置的dataDir目录中创建一个myid文件，里面内容为一个数字，用来标识当前主机，$ZOOKEEPER_HOME/conf/zoo.cfg文件中配置的server.X，则myid文件中就输入这个数字X。（即在每个节点上新建并设置文件myid，其内容与zoo.cfg中的id相对应）</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">cd</span> /home/hadoop/cloud/zookeeper-3.4.9/data/</span><br><span class="line">mkdir myid</span><br><span class="line">vi myid</span><br></pre></td></tr></table></figure>
</li>
<li><p>启动zookeeper</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="variable">$ZOOKEEPER_HOME</span>/bin/zkServer.sh start</span><br></pre></td></tr></table></figure>
</li>
</ul>
<h3 id="安装storm"><a href="#安装storm" class="headerlink" title="安装storm"></a>安装storm</h3><ul>
<li>下载安装包，解压到 /home/hadoop/cloud 目录下</li>
<li>设置环境变量</li>
<li><p>修改 storm 主目录下conf/storm.yaml文件</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">storm.zookeeper.servers:</span><br><span class="line"> - <span class="string">"zk01"</span></span><br><span class="line"> - <span class="string">"zk02"</span></span><br><span class="line"> - <span class="string">"zk03"</span></span><br><span class="line">storm.zookeeper.port: 2181</span><br><span class="line">storm.zookeeper.root: <span class="string">"/storm"</span></span><br><span class="line">nimbus.host: <span class="string">"storm01"</span></span><br><span class="line"></span><br><span class="line">supervisor.slots.ports:</span><br><span class="line"> - 6700</span><br><span class="line"> - 6701</span><br><span class="line"> - 6702</span><br><span class="line"> - 6703</span><br><span class="line"> - 6704</span><br><span class="line"> - 6705</span><br><span class="line"> - 6706</span><br><span class="line"> - 6707</span><br><span class="line"> - 6708</span><br><span class="line">storm.local.dir: <span class="string">"/home/hadoop/cloud/apache-storm-0.9.6/local"</span></span><br><span class="line"></span><br><span class="line">drpc.servers:</span><br><span class="line"> - <span class="string">"storm01"</span></span><br><span class="line">drpc.port: 3772</span><br><span class="line">drpc.worker.threads: 100</span><br><span class="line">drpc.queue.size: 1024</span><br><span class="line">drpc.invocations.port: 3773</span><br><span class="line">drpc.request.timeout.secs: 800</span><br><span class="line">drpc.childopts: <span class="string">"-Xmx2048m"</span></span><br><span class="line">drpc.port: 3772</span><br><span class="line">drpc.worker.threads: 100</span><br><span class="line">drpc.queue.size: 1024</span><br><span class="line">drpc.invocations.port: 3773</span><br><span class="line">drpc.request.timeout.secs: 800</span><br><span class="line">drpc.childopts: <span class="string">"-Xmx2048m"</span></span><br><span class="line"></span><br><span class="line"><span class="comment">#JVM options of worker</span></span><br><span class="line"><span class="comment">#edited by persist</span></span><br><span class="line">superivisor.childopts: <span class="string">"-Xmx2048m"</span></span><br><span class="line">worker.childopts: <span class="string">"-Xmx2048m"</span></span><br><span class="line"><span class="comment">#JVM options of worker</span></span><br><span class="line"><span class="comment">#edited by persist</span></span><br><span class="line">superivisor.childopts: <span class="string">"-Xmx2048m"</span></span><br><span class="line">worker.childopts: <span class="string">"-Xmx2048m"</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>启动storm命令（<strong>启动storm之前需要启动zookeeper</strong>）</p>
</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">nimubs-host: nohup bin/storm nimbus&amp;</span><br><span class="line">nimubs-host: nohup bin/storm ui&amp;</span><br><span class="line">superivisor-hosts: nohup bin/storm superivisor&amp;</span><br></pre></td></tr></table></figure>
<ul>
<li>浏览器输入：<a href="http://pc1080:8080" target="_blank" rel="noopener">http://pc1080:8080</a> 查看storm UI界面</li>
</ul>
<h3 id="安装hadoop"><a href="#安装hadoop" class="headerlink" title="安装hadoop"></a>安装hadoop</h3><ul>
<li>下载安装包，解压到 /home/hadoop/cloud 目录下</li>
<li>设置环境变量</li>
<li><p>修改主目录下 etc/hadoop-env.sh</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> JAVA_HOME=/home/hadoop/cloud/jdk1.8.0_92/</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改主目录下 etc/yarn-env.sh</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> JAVA_HOME=/home/hadoop/cloud/jdk1.8.0_92/</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改 etc/hadoop/mapred-env.sh</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> JAVA_HOME=/home/hadoop/cloud/jdk1.8.0_92/</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改主目录下 etc/core-site.xml</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">            &lt;name&gt;fs.defaultFS&lt;/name&gt;</span><br><span class="line">            &lt;value&gt;hdfs://hadoop01:9000&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">                &lt;name&gt;hadoop.tmp.dir&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;file:/home/hadoop/cloud/hadoop-2.6.4/data/tmp&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;Abase <span class="keyword">for</span> other temporary directories.&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;ha.zookeeper.quorum&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;zk01:2181,zk02:2181,zk03:2181&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;这里是ZooKeeper集群的地址和端口。注意，数量一定是奇数，且不少于三个节点&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    <span class="comment"># add ours here</span></span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.proxyuser.hadoop.groups&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;*&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;允许所有用户组用户代理&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hadoop.proxyuser.hadoop.hosts&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hadoop01&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;允许挂载的主机域名&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改主目录下 etc/hdfs-site.xml</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.nfs.exports.allowed.hosts&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;hadoop01 rw;hadoop02 rw;hadoop03 rw;&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.secondary.http-address&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;hadoop01:50090&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;2&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.name.dir&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;file:/home/hadoop/cloud/hadoop-2.6.4/tmp/dfs/name&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.datanode.data.dir&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;file:/home/hadoop/cloud/hadoop-2.6.4/tmp/dfs/data&lt;/value&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">		&lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.webhdfs.enabled&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">		&lt;/property&gt;</span><br><span class="line"></span><br><span class="line">        <span class="comment">## add options here</span></span><br><span class="line">		&lt;property&gt;</span><br><span class="line">				&lt;name&gt;dfs.client.block.write.replace-datanode-on-failure.enable&lt;/name&gt;</span><br><span class="line">				&lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">		&lt;/property&gt;</span><br><span class="line">		&lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.client.block.write.replace-datanode-on-failure.policy&lt;/name&gt;</span><br><span class="line">		        &lt;value&gt;NEVER&lt;/value&gt;</span><br><span class="line">		&lt;/property&gt;</span><br><span class="line">		&lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.fs-limits.max-component-length&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;255&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;Defines the maximum number of bytes <span class="keyword">in</span> UTF-8 encoding <span class="keyword">in</span> each</span><br><span class="line">                    component of a path.  A value of 0 will <span class="built_in">disable</span> the check.&lt;/description&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.fs-limits.max-directory-items&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;6000000&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;Defines the maximum number of items that a directory may</span><br><span class="line">                    contain. Cannot <span class="built_in">set</span> the property to a value less than 1 or more than</span><br><span class="line">                    6400000.&lt;/description&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.fs-limits.min-block-size&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;1048576&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;Minimum block size <span class="keyword">in</span> bytes, enforced by the Namenode at create</span><br><span class="line">                    time. This prevents the accidental creation of files with tiny block</span><br><span class="line">                    sizes (and thus many blocks), <span class="built_in">which</span> can degrade performance.&lt;/description&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.fs-limits.max-blocks-per-file&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;1048576&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;Maximum number of blocks per file, enforced by the Namenode on</span><br><span class="line">                    write. This prevents the creation of extremely large files <span class="built_in">which</span> can</span><br><span class="line">                    degrade performance.&lt;/description&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.fs-limits.max-xattrs-per-inode&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;32&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;</span><br><span class="line">                    Maximum number of extended attributes per inode.</span><br><span class="line">                &lt;/description&gt;</span><br><span class="line">        &lt;/property&gt;</span><br><span class="line">        &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.namenode.fs-limits.max-xattr-size&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;16384&lt;/value&gt;</span><br><span class="line">                &lt;description&gt;</span><br><span class="line">                    The maximum combined size of the name and value of an extended attribute</span><br><span class="line">                    <span class="keyword">in</span> bytes. It should be larger than 0, and less than or equal to maximum</span><br><span class="line">                    size hard <span class="built_in">limit</span> <span class="built_in">which</span> is 32768.</span><br><span class="line">                &lt;/description&gt;</span><br><span class="line">       &lt;/property&gt;</span><br><span class="line">	   &lt;property&gt;</span><br><span class="line">                &lt;name&gt;dfs.datanode.max.transfer.threads&lt;/name&gt;</span><br><span class="line">                &lt;value&gt;8192&lt;/value&gt;</span><br><span class="line">      &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改主目录下 etc/httpfs-env.sh</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> JAVA_HOME=/home/hadoop/cloud/jdk1.8.0_92/</span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HTTPFS_HOME=/home/hadoop/cloud/hadoop-2.6.4/</span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HTTPFS_CONFIG=/home/hadoop/cloud/hadoop-2.6.4/etc/hadoop</span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> CATALINA_BASE=/home/hadoop/cloud/hadoop-2.6.4/share/hadoop/httpfs/tomcat</span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HTTPFS_LOG=<span class="variable">$&#123;HTTPFS_HOME&#125;</span>/httpfs/logs</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HTTPFS_TEMP=<span class="variable">$&#123;HTTPFS_HOME&#125;</span>/httpfs/temp</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HTTPFS_HTTP_PORT=14000</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="built_in">export</span> HTTPFS_ADMIN_PORT=`expr <span class="variable">$&#123;HTTPFS_HTTP_PORT&#125;</span> + 1`</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改 etc/hadoop/slavers</p>
</li>
</ul>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">hadoop01</span><br><span class="line">hadoop02</span><br><span class="line">hadoop03</span><br></pre></td></tr></table></figure>
<p><span style="color:red">以上只是部分配置，其中hadoop01是默认的主节点，默认配置的主机</span></p>
<ul>
<li><p>启动hadoop</p>
<ul>
<li>首先需要启动zookeeper</li>
<li>格式化hdfs： bin/hdfs namenode -format</li>
<li>启动hdfs和yarn： sbin/start-all.sh,启动成功master出现NameNode，SecondaryNameNode，ResourceManger进程，slave节点出现DataNode和NodeManager<br><strong>注意事项：如果增加了多个ip映射，那么有可能提醒是否需要添加host到ssh表，这种情况框需要重新设置ssh互通</strong></li>
</ul>
</li>
<li><p>web查看<br>hdfs: <a href="https://hadoop01:50070/" target="_blank" rel="noopener">https://hadoop01:50070/</a><br>yarn: <a href="https://hadoop01:8088/" target="_blank" rel="noopener">https://hadoop01:8088/</a></p>
</li>
</ul>
<p>遇到的问题：<br>主节点上启动start-all.sh, 只有主节点的NameNode和SecondaryNameNode和DataNode启动，从节点的DataNode没有启动</p>
<p>重启部分坏死节点：<br>bin/Hadoop-daemon.sh start DataNode<br>bin/Hadoop-daemon.sh start jobtracker</p>
<p>动态加入新节点：<br>bin/Hadoop-daemon.sh —config ./conf start DataNode<br>bin/Hadoop-daemon.sh —config ./conf start tasktracker</p>
<p>更新配置文件命令：<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> ip <span class="keyword">in</span> `seq 2 3`; <span class="keyword">do</span> scp /home/hadoop/cloud/hadoop-2.6.4/etc/hadoop/* hadoop@hadoop0<span class="variable">$ip</span>:/home/hadoop/cloud/hadoop-2.6.4/etc/hadoop/; <span class="keyword">done</span></span><br></pre></td></tr></table></figure></p>
<p>更新环境变量：<br><figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> ip <span class="keyword">in</span> `seq 2 3`; <span class="keyword">do</span> scp /home/hadoop/.bashrc hadoop@hadoop0<span class="variable">$ip</span>:/home/hadoop/; <span class="keyword">done</span></span><br></pre></td></tr></table></figure></p>
<h3 id="安装HBase"><a href="#安装HBase" class="headerlink" title="安装HBase"></a>安装HBase</h3><ul>
<li>下载安装包，解压到 /home/hadoop/cloud 目录下</li>
<li>修改～/.bashrc文件，添加HBASE_HOME环境变量；</li>
<li><p>修改 conf/hbase-env.sh</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> JAVA_HOME=/home/hadoop/cloud/jdk1.8.0_92/</span><br><span class="line"><span class="built_in">export</span> HBASE_CLASSPATH=/opt/hbase/conf  (extral classpath like java, Optional)</span><br><span class="line"><span class="comment"># 此配置信息，设置由zk集群管理，故为false  </span></span><br><span class="line"><span class="built_in">export</span> HBASE_MANAGES_ZK=<span class="literal">false</span>  </span><br><span class="line"><span class="built_in">export</span> HBASE_HOME=/opt/hbase  (Optional)</span><br><span class="line"><span class="comment">#Hbase日志目录  </span></span><br><span class="line"><span class="built_in">export</span> HBASE_LOG_DIR=<span class="variable">$HBASE_HOME</span>/logs(Optional)</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改 conf/hbase-site.xml</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br></pre></td><td class="code"><pre><span class="line">&lt;configuration&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.rootdir&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hdfs://hbase01:9000/hbase&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;The directory shared by region servers.&lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.zookeeper.property.clientPort&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;2181&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;Property from ZooKeeper config zoo.cfg. The port at <span class="built_in">which</span> the clients will connect.</span><br><span class="line">        &lt;/description&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.master.info.bindAddress&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;192.168.100.101&lt;/value&gt;</span><br><span class="line">        &lt;description&gt;HBase Master web ui <span class="built_in">bind</span> address&lt;/description&gt; **必须填写master节点地址**</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;zookeeper.session.timeout&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;60000&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.zookeeper.quorum&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;hbase01,hbase02,hbase03&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.tmp.dir&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;/home/hadoop/cloud/hbase-1.2.4/data/tmp&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.cluster.distributed&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.zookeeper.property.dataDir&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;/home/hadoop/cloud/hbase-1.2.4/zookeeper&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;hbase.zookeeper.property.maxClientCnxns&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;300&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;dfs.client.block.write.replace-datanode-on-failure.enable&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;<span class="literal">true</span>&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;dfs.client.block.write.replace-datanode-on-failure.policy&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;NEVER&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">    &lt;property&gt;</span><br><span class="line">        &lt;name&gt;dfs.replication&lt;/name&gt;</span><br><span class="line">        &lt;value&gt;1&lt;/value&gt;</span><br><span class="line">    &lt;/property&gt;</span><br><span class="line">&lt;/configuration&gt;</span><br></pre></td></tr></table></figure>
</li>
<li><p>修改 regionservers</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">hbase01</span><br><span class="line">hbase02  <span class="comment">#设置regionserver的节点</span></span><br><span class="line">hbase03</span><br></pre></td></tr></table></figure>
</li>
<li><p>同步配置</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">for</span> ip <span class="keyword">in</span> `seq 2 3`; <span class="keyword">do</span> scp /home/hadoop/cloud/hbase-1.2.4/conf/* hadoop@hadoop0<span class="variable">$ip</span>:/home/hadoop/cloud/hbase-1.2.4/conf/; <span class="keyword">done</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>启动hbase</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">bin/start-hbase.sh</span><br><span class="line">启动后，jps可以看到有这两个进程</span><br><span class="line">HRegionServer</span><br><span class="line">HMaster</span><br></pre></td></tr></table></figure>
</li>
<li><p>启动Hbase Shell</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">bin/hbase shell</span><br></pre></td></tr></table></figure>
</li>
<li><p>访问HBase UI<br>查看hbase管理界面<a href="http://192.168.181.66:16010" target="_blank" rel="noopener">http://192.168.181.66:16010</a></p>
</li>
<li><p>hbase日常维护<br>参考链接：<a href="https://xiongraorao.github.io/2017/03/09/Hbase%E5%9F%BA%E6%9C%AC%E6%93%8D%E4%BD%9C%E5%91%BD%E4%BB%A4/#more">Hbase基本操作命令</a></p>
</li>
</ul>
<blockquote>
<p><span style="color:red;font-size: 26px;font-weight: bold">注意hbase集群安装的大坑</span></p>
<ol>
<li>hbase-site.xml必须添加hbase.master.info.bindAddress配置项，设置为master节点的ip地址</li>
<li>设置/etc/hosts文件，注释掉 127.0.1.1 hostname这一行，替换成局域网ip hostname，否则HRegionServer 无法链接Master节点</li>
</ol>
</blockquote>
<p>错误参考链接：<a href="http://www.cnblogs.com/colorfulkoala/archive/2012/07/09/2583841.html" target="_blank" rel="noopener">http://www.cnblogs.com/colorfulkoala/archive/2012/07/09/2583841.html</a></p>
<h3 id="安装kafka"><a href="#安装kafka" class="headerlink" title="安装kafka"></a>安装kafka</h3><p>参考链接：<a href="http://121.42.164.108/index.php/2017/03/08/ubuntu14-04-stormkafkazookeeperhdfs/" target="_blank" rel="noopener">kafka</a></p>

      
    </div>

    

    
    
    

    

    

    

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/caffe/" rel="tag"># caffe</a>
          
            <a href="/tags/深度学习/" rel="tag"># 深度学习</a>
          
            <a href="/tags/大数据/" rel="tag"># 大数据</a>
          
            <a href="/tags/cuda/" rel="tag"># cuda</a>
          
            <a href="/tags/opencv/" rel="tag"># opencv</a>
          
            <a href="/tags/分布式/" rel="tag"># 分布式</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/2017/03/03/Ubuntu制作UbuntuLiveCD-备份系统镜像教程/" rel="next" title="Ubuntu制作UbuntuLiveCD+备份系统镜像教程">
                <i class="fa fa-chevron-left"></i> Ubuntu制作UbuntuLiveCD+备份系统镜像教程
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/2017/03/09/Hbase基本操作命令/" rel="prev" title="Hbase基本操作命令">
                Hbase基本操作命令 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
        
  <div class="bdsharebuttonbox">
    <a href="#" class="bds_tsina" data-cmd="tsina" title="分享到新浪微博"></a>
    <a href="#" class="bds_douban" data-cmd="douban" title="分享到豆瓣网"></a>
    <a href="#" class="bds_sqq" data-cmd="sqq" title="分享到QQ好友"></a>
    <a href="#" class="bds_qzone" data-cmd="qzone" title="分享到QQ空间"></a>
    <a href="#" class="bds_weixin" data-cmd="weixin" title="分享到微信"></a>
    <a href="#" class="bds_tieba" data-cmd="tieba" title="分享到百度贴吧"></a>
    <a href="#" class="bds_twi" data-cmd="twi" title="分享到Twitter"></a>
    <a href="#" class="bds_fbook" data-cmd="fbook" title="分享到Facebook"></a>
    <a href="#" class="bds_more" data-cmd="more"></a>
    <a class="bds_count" data-cmd="count"></a>
  </div>
  <script>
    window._bd_share_config = {
      "common": {
        "bdText": "",
        "bdMini": "2",
        "bdMiniList": false,
        "bdPic": ""
      },
      "share": {
        "bdSize": "16",
        "bdStyle": "0"
      },
      "image": {
        "viewList": ["tsina", "douban", "sqq", "qzone", "weixin", "twi", "fbook"],
        "viewText": "分享到：",
        "viewSize": "16"
      }
    }
  </script>

<script>
  <!-- with(document)0[(getElementsByTagName('head')[0]||body).appendChild(createElement('script')).src='//bdimg.share.baidu.com/static/api/js/share.js?cdnversion='+~(-new Date()/36e5)]; -->
  with(document)0[(getElementsByTagName('head')[0]||body).appendChild(createElement('script')).src='/static/api/js/share.js?v=89860593.js?cdnversion='+~(-new Date()/36e5)];
</script>

      
    </div>
  </div>


          </div>
          

  
    <div class="comments" id="comments">
      
        <div id="gitment-container"></div>
      
    </div>

  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image"
                src="/uploads/avatar.jpg"
                alt="Xiong Raorao" />
            
              <p class="site-author-name" itemprop="name">Xiong Raorao</p>
              <p class="site-description motion-element" itemprop="description">用我的知识改变世界</p>
          </div>

          
            <nav class="site-state motion-element">
              
                <div class="site-state-item site-state-posts">
                
                  <a href="/archives/">
                
                    <span class="site-state-item-count">43</span>
                    <span class="site-state-item-name">日志</span>
                  </a>
                </div>
              

              
                
                
                <div class="site-state-item site-state-categories">
                  <a href="/categories/index.html">
                    
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                    <span class="site-state-item-count">24</span>
                    <span class="site-state-item-name">分类</span>
                  </a>
                </div>
              

              
                
                
                <div class="site-state-item site-state-tags">
                  <a href="/tags/index.html">
                    
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                      
                    
                    <span class="site-state-item-count">42</span>
                    <span class="site-state-item-name">标签</span>
                  </a>
                </div>
              
            </nav>
          

          
            <div class="feed-link motion-element">
              <a href="/atom.xml" rel="alternate">
                <i class="fa fa-rss"></i>
                RSS
              </a>
            </div>
          

          
            <div class="links-of-author motion-element">
              
                <span class="links-of-author-item">
                  <a href="https://github.com/xiongraorao" target="_blank" title="GitHub" rel="external nofollow"><i class="fa fa-fw fa-github"></i>GitHub</a>
                  
                </span>
              
                <span class="links-of-author-item">
                  <a href="mailto:xiongraorao@gmail.com" target="_blank" title="E-Mail" rel="external nofollow"><i class="fa fa-fw fa-envelope"></i>E-Mail</a>
                  
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://plus.google.com/109510977952432239538" target="_blank" title="Google" rel="external nofollow"><i class="fa fa-fw fa-google"></i>Google</a>
                  
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://weibo.com/u/3605456425" target="_blank" title="微博" rel="external nofollow"><i class="fa fa-fw fa-weibo"></i>微博</a>
                  
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://www.douban.com/people/139800623/" target="_blank" title="豆瓣" rel="external nofollow"><i class="fa fa-fw fa-globe"></i>豆瓣</a>
                  
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/xiong-ge-78/" target="_blank" title="知乎" rel="external nofollow"><i class="fa fa-fw fa-globe"></i>知乎</a>
                  
                </span>
              
                <span class="links-of-author-item">
                  <a href="https://twitter.com/_Paul_forever" target="_blank" title="Twitter" rel="external nofollow"><i class="fa fa-fw fa-twitter"></i>Twitter</a>
                  
                </span>
              
            </div>
          

          
          

          
          

          
            
          
          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#安装ubuntu"><span class="nav-number">1.</span> <span class="nav-text">安装ubuntu</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装NVIDIA驱动"><span class="nav-number">2.</span> <span class="nav-text">安装NVIDIA驱动</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装cuda-cudnn-opencv-caffe"><span class="nav-number">3.</span> <span class="nav-text">安装cuda+cudnn+opencv+caffe</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#配置静态ip-安装ssh-server"><span class="nav-number">4.</span> <span class="nav-text">配置静态ip + 安装ssh-server</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#静态ip"><span class="nav-number">4.1.</span> <span class="nav-text">静态ip</span></a></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#network-192-168-0-0"><span class="nav-number"></span> <span class="nav-text">network 192.168.0.0</span></a></li><li class="nav-item nav-level-1"><a class="nav-link" href="#broadcast-192-168-0-255"><span class="nav-number"></span> <span class="nav-text">broadcast 192.168.0.255</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#安装ssh-server"><span class="nav-number">0.1.</span> <span class="nav-text">安装ssh-server</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#安装storm-kafka-zookeeper-hdfs-hbase-jdk"><span class="nav-number">1.</span> <span class="nav-text">安装storm+kafka+zookeeper+hdfs+hbase+jdk</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#前期准备工作"><span class="nav-number">1.1.</span> <span class="nav-text">前期准备工作</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装zookeeper"><span class="nav-number">1.2.</span> <span class="nav-text">安装zookeeper</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装storm"><span class="nav-number">1.3.</span> <span class="nav-text">安装storm</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装hadoop"><span class="nav-number">1.4.</span> <span class="nav-text">安装hadoop</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装HBase"><span class="nav-number">1.5.</span> <span class="nav-text">安装HBase</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#安装kafka"><span class="nav-number">1.6.</span> <span class="nav-text">安装kafka</span></a></li></ol></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; 2016 &mdash; <span itemprop="copyrightYear">2018</span>
  <span class="with-love" id="animate">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Xiong Raorao</span>

  

  
</div>


  



  <div class="powered-by">由 <a class="theme-link" target="_blank" rel="external nofollow" href="https://hexo.io">Hexo</a> 强力驱动 v3.7.1</div>



  <span class="post-meta-divider">|</span>



  <div class="theme-info">主题 &mdash; <a class="theme-link" target="_blank" rel="external nofollow" href="https://github.com/theme-next/hexo-theme-next">NexT.Pisces</a> v6.3.0</div>




        








        
      </div>
    </footer>

    
      <div class="back-to-top">
        <i class="fa fa-arrow-up"></i>
        
      </div>
    

    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>














  



  
  



  
  



  
  





  



  
  
    <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/lib/canvas-nest/canvas-nest.min.js"></script>
  

  
  
    <script type="text/javascript" src="/lib/three/three.min.js"></script>
  

  
  
    <script type="text/javascript" src="/lib/three/three-waves.min.js"></script>
  

  
  
    <script type="text/javascript" src="/lib/three/canvas_lines.min.js"></script>
  

  
  
    <script type="text/javascript" src="/lib/three/canvas_sphere.min.js"></script>
  

  
  
    <script type="text/javascript" src="/lib/reading_progress/reading_progress.js"></script>
  


  


  <script type="text/javascript" src="/js/src/utils.js?v=6.3.0"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=6.3.0"></script>



  
  


  <script type="text/javascript" src="/js/src/affix.js?v=6.3.0"></script>

  <script type="text/javascript" src="/js/src/schemes/pisces.js?v=6.3.0"></script>



  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=6.3.0"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=6.3.0"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=6.3.0"></script>



  



	





  





  






<!-- LOCAL: You can save these files to your site and update links -->
    
        
        <link rel="stylesheet" href="https://aimingoo.github.io/gitmint/style/default.css">
        <script src="https://aimingoo.github.io/gitmint/dist/gitmint.browser.js"></script>
    
<!-- END LOCAL -->

    

    
      <script type="text/javascript">
      function renderGitment(){
        var gitment = new Gitmint({
            id: window.location.pathname,
            owner: 'xiongraorao',
            repo: 'hexo-comment',
            
            lang: "" || navigator.language || navigator.systemLanguage || navigator.userLanguage,
            
            oauth: {
            
            
                client_secret: 'f5b4b12d0ce3eb0bf7e9294eed6726f836f43ded',
            
                client_id: 'b26d67370855fd2d2294'
            }});
        gitment.render('gitment-container');
      }

      
      renderGitment();
      
      </script>
    






  

  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    var isXml = true;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length === 0) {
      search_path = "search.xml";
    } else if (/json$/i.test(search_path)) {
      isXml = false;
    }
    var path = "/" + search_path;
    // monitor main search box;

    var onPopupClose = function (e) {
      $('.popup').hide();
      $('#local-search-input').val('');
      $('.search-result-list').remove();
      $('#no-result').remove();
      $(".local-search-pop-overlay").remove();
      $('body').css('overflow', '');
    }

    function proceedsearch() {
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay"></div>')
        .css('overflow', 'hidden');
      $('.search-popup-overlay').click(onPopupClose);
      $('.popup').toggle();
      var $localSearchInput = $('#local-search-input');
      $localSearchInput.attr("autocapitalize", "none");
      $localSearchInput.attr("autocorrect", "off");
      $localSearchInput.focus();
    }

    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';

      // start loading animation
      $("body")
        .append('<div class="search-popup-overlay local-search-pop-overlay">' +
          '<div id="search-loading-icon">' +
          '<i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>' +
          '</div>' +
          '</div>')
        .css('overflow', 'hidden');
      $("#search-loading-icon").css('margin', '20% auto 0 auto').css('text-align', 'center');

      

      $.ajax({
        url: path,
        dataType: isXml ? "xml" : "json",
        async: true,
        success: function(res) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = isXml ? $("entry", res).map(function() {
            return {
              title: $("title", this).text(),
              content: $("content",this).text(),
              url: $("url" , this).text()
            };
          }).get() : res;
          var input = document.getElementById(search_id);
          var resultContent = document.getElementById(content_id);
          var inputEventFunction = function() {
            var searchText = input.value.trim().toLowerCase();
            var keywords = searchText.split(/[\s\-]+/);
            if (keywords.length > 1) {
              keywords.push(searchText);
            }
            var resultItems = [];
            if (searchText.length > 0) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var hitCount = 0;
                var searchTextCount = 0;
                var title = data.title.trim();
                var titleInLowerCase = title.toLowerCase();
                var content = data.content.trim().replace(/<[^>]+>/g,"");
                
                var contentInLowerCase = content.toLowerCase();
                var articleUrl = decodeURIComponent(data.url);
                var indexOfTitle = [];
                var indexOfContent = [];
                // only match articles with not empty titles
                if(title != '') {
                  keywords.forEach(function(keyword) {
                    function getIndexByWord(word, text, caseSensitive) {
                      var wordLen = word.length;
                      if (wordLen === 0) {
                        return [];
                      }
                      var startPosition = 0, position = [], index = [];
                      if (!caseSensitive) {
                        text = text.toLowerCase();
                        word = word.toLowerCase();
                      }
                      while ((position = text.indexOf(word, startPosition)) > -1) {
                        index.push({position: position, word: word});
                        startPosition = position + wordLen;
                      }
                      return index;
                    }

                    indexOfTitle = indexOfTitle.concat(getIndexByWord(keyword, titleInLowerCase, false));
                    indexOfContent = indexOfContent.concat(getIndexByWord(keyword, contentInLowerCase, false));
                  });
                  if (indexOfTitle.length > 0 || indexOfContent.length > 0) {
                    isMatch = true;
                    hitCount = indexOfTitle.length + indexOfContent.length;
                  }
                }

                // show search results

                if (isMatch) {
                  // sort index by position of keyword

                  [indexOfTitle, indexOfContent].forEach(function (index) {
                    index.sort(function (itemLeft, itemRight) {
                      if (itemRight.position !== itemLeft.position) {
                        return itemRight.position - itemLeft.position;
                      } else {
                        return itemLeft.word.length - itemRight.word.length;
                      }
                    });
                  });

                  // merge hits into slices

                  function mergeIntoSlice(text, start, end, index) {
                    var item = index[index.length - 1];
                    var position = item.position;
                    var word = item.word;
                    var hits = [];
                    var searchTextCountInSlice = 0;
                    while (position + word.length <= end && index.length != 0) {
                      if (word === searchText) {
                        searchTextCountInSlice++;
                      }
                      hits.push({position: position, length: word.length});
                      var wordEnd = position + word.length;

                      // move to next position of hit

                      index.pop();
                      while (index.length != 0) {
                        item = index[index.length - 1];
                        position = item.position;
                        word = item.word;
                        if (wordEnd > position) {
                          index.pop();
                        } else {
                          break;
                        }
                      }
                    }
                    searchTextCount += searchTextCountInSlice;
                    return {
                      hits: hits,
                      start: start,
                      end: end,
                      searchTextCount: searchTextCountInSlice
                    };
                  }

                  var slicesOfTitle = [];
                  if (indexOfTitle.length != 0) {
                    slicesOfTitle.push(mergeIntoSlice(title, 0, title.length, indexOfTitle));
                  }

                  var slicesOfContent = [];
                  while (indexOfContent.length != 0) {
                    var item = indexOfContent[indexOfContent.length - 1];
                    var position = item.position;
                    var word = item.word;
                    // cut out 100 characters
                    var start = position - 20;
                    var end = position + 80;
                    if(start < 0){
                      start = 0;
                    }
                    if (end < position + word.length) {
                      end = position + word.length;
                    }
                    if(end > content.length){
                      end = content.length;
                    }
                    slicesOfContent.push(mergeIntoSlice(content, start, end, indexOfContent));
                  }

                  // sort slices in content by search text's count and hits' count

                  slicesOfContent.sort(function (sliceLeft, sliceRight) {
                    if (sliceLeft.searchTextCount !== sliceRight.searchTextCount) {
                      return sliceRight.searchTextCount - sliceLeft.searchTextCount;
                    } else if (sliceLeft.hits.length !== sliceRight.hits.length) {
                      return sliceRight.hits.length - sliceLeft.hits.length;
                    } else {
                      return sliceLeft.start - sliceRight.start;
                    }
                  });

                  // select top N slices in content

                  var upperBound = parseInt('1');
                  if (upperBound >= 0) {
                    slicesOfContent = slicesOfContent.slice(0, upperBound);
                  }

                  // highlight title and content

                  function highlightKeyword(text, slice) {
                    var result = '';
                    var prevEnd = slice.start;
                    slice.hits.forEach(function (hit) {
                      result += text.substring(prevEnd, hit.position);
                      var end = hit.position + hit.length;
                      result += '<b class="search-keyword">' + text.substring(hit.position, end) + '</b>';
                      prevEnd = end;
                    });
                    result += text.substring(prevEnd, slice.end);
                    return result;
                  }

                  var resultItem = '';

                  if (slicesOfTitle.length != 0) {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + highlightKeyword(title, slicesOfTitle[0]) + "</a>";
                  } else {
                    resultItem += "<li><a href='" + articleUrl + "' class='search-result-title'>" + title + "</a>";
                  }

                  slicesOfContent.forEach(function (slice) {
                    resultItem += "<a href='" + articleUrl + "'>" +
                      "<p class=\"search-result\">" + highlightKeyword(content, slice) +
                      "...</p>" + "</a>";
                  });

                  resultItem += "</li>";
                  resultItems.push({
                    item: resultItem,
                    searchTextCount: searchTextCount,
                    hitCount: hitCount,
                    id: resultItems.length
                  });
                }
              })
            };
            if (keywords.length === 1 && keywords[0] === "") {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>'
            } else if (resultItems.length === 0) {
              resultContent.innerHTML = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>'
            } else {
              resultItems.sort(function (resultLeft, resultRight) {
                if (resultLeft.searchTextCount !== resultRight.searchTextCount) {
                  return resultRight.searchTextCount - resultLeft.searchTextCount;
                } else if (resultLeft.hitCount !== resultRight.hitCount) {
                  return resultRight.hitCount - resultLeft.hitCount;
                } else {
                  return resultRight.id - resultLeft.id;
                }
              });
              var searchResultList = '<ul class=\"search-result-list\">';
              resultItems.forEach(function (result) {
                searchResultList += result.item;
              })
              searchResultList += "</ul>";
              resultContent.innerHTML = searchResultList;
            }
          }

          if ('auto' === 'auto') {
            input.addEventListener('input', inputEventFunction);
          } else {
            $('.search-icon').click(inputEventFunction);
            input.addEventListener('keypress', function (event) {
              if (event.keyCode === 13) {
                inputEventFunction();
              }
            });
          }

          // remove loading animation
          $(".local-search-pop-overlay").remove();
          $('body').css('overflow', '');

          proceedsearch();
        }
      });
    }

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched === false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });

    $('.popup-btn-close').click(onPopupClose);
    $('.popup').click(function(e){
      e.stopPropagation();
    });
    $(document).on('keyup', function (event) {
      var shouldDismissSearchPopup = event.which === 27 &&
        $('.search-popup').is(':visible');
      if (shouldDismissSearchPopup) {
        onPopupClose();
      }
    });
  </script>





  

  
  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.4.js"></script>
  <script>AV.initialize("Nobvpla4KCLkSofVkh4r7KQt-gzGzoHsz", "vmdldWBq3N8oOWIbDrajVMFE");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for(var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if( countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            
            counter.save(null, {
              success: function(counter) {
                
                  var $element = $(document.getElementById(url));
                  $element.find('.leancloud-visitors-count').text(counter.get('time'));
                
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            
              var newcounter = new Counter();
              /* Set ACL */
              var acl = new AV.ACL();
              acl.setPublicReadAccess(true);
              acl.setPublicWriteAccess(true);
              newcounter.setACL(acl);
              /* End Set ACL */
              newcounter.set("title", title);
              newcounter.set("url", url);
              newcounter.set("time", 1);
              newcounter.save(null, {
                success: function(newcounter) {
                  var $element = $(document.getElementById(url));
                  $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
                },
                error: function(newcounter, error) {
                  console.log('Failed to create');
                }
              });
            
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script>



  

  
<script>
(function(){
    var bp = document.createElement('script');
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https') {
        bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';        
    }
    else {
        bp.src = 'http://push.zhanzhang.baidu.com/push.js';
    }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
})();
</script>


  

  
  

  

  
  <script type="text/javascript" src="/js/src/js.cookie.js?v=6.3.0"></script>
  <script type="text/javascript" src="/js/src/scroll-cookie.js?v=6.3.0"></script>


  

  

  

</body>
</html>
